# -*- coding: utf-8 -*-
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from jdspider.items import JdspiderItem
import re
import requests
import json


class JdSpider(CrawlSpider):
    name = 'jd'
    allowed_domains = ['jd.com']
    start_urls = [
        'http://channel.jd.com/fashion.html',
    ]

    rules = (
        Rule(LinkExtractor(allow=r'channel.jd.com/.*?.html'), follow=True),
        Rule(LinkExtractor(allow=r'search.jd.com/Search\?keyword=.*'), follow=True),
        Rule(LinkExtractor(allow=r'item.jd.com/\d*?.html'), callback='parse_item', follow=True),
    )

    def parse_item(self, response):
        item = JdspiderItem()

        # 品牌
        item['brand'] = re.compile(r"<li.*?>品牌.*?>(.*?)</a>").findall(response.body.decode('GBK', 'ignore'))[0]
        # 商品名
        item['name'] = re.compile(r"<li.*?>商品名称：(.*?)</li>").findall(response.body.decode('GBK', 'ignore'))[0]
        # 价格
        item['price'] = self.get_price(response)
        # 评论数
        item['comments'] = self.get_comments(response)
        # 大分类
        item['big_class'] = re.compile(r'<a.*?mbNav-2">(.*?)</a></div>').findall(response.body.decode('GBK', 'ignore'))[0]
        # 小分类
        item['small_class'] = re.compile(r'<a.*?mbNav-3">(.*?)</a></div>').findall(response.body.decode('GBK', 'ignore'))[0]
        # 适用人群
        item['apply'] = self.get_apply(response)
        # 上市时间
        item['listed_time'] = self.get_listed_time(response)
        # 链接
        item['link'] = response.url

        yield item

    def get_price(self, response):
        """获取价格并返回"""
        # 取得 skuids 的值
        sku_id = re.compile(r'skuid: (\d*?),').findall(response.body.decode('GBK', 'ignore'))[0]

        # 获取价格
        url = 'http://wq.jd.com/pingou_api/BatGetPingouInfo?skuids={}&source=pc'.format(sku_id)
        headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:50.0) Gecko/20100101 Firefox/50.0',
            'Host': 'wq.jd.com',
            'Referer': 'http://item.jd.com/33041828758.html',
        }

        res = requests.get(url=url, headers=headers)
        data_js = json.loads(res.text)

        return data_js['pingou_info'][0]['bp'] if data_js['pingou_info'][0]['bp'] else data_js['pingou_info'][0]['p']

    def get_comments(self, response):
        """获取评论数并返回"""
        # 取得 skuids 的值
        sku_id = re.compile(r'skuid: (\d*?),').findall(response.body.decode('GBK', 'ignore'))[0]

        # 获取评论数并返回
        url = 'http://club.jd.com/comment/productCommentSummaries.action?referenceIds={}'.format(sku_id)
        headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:50.0) Gecko/20100101 Firefox/50.0',
            'Host': 'club.jd.com',
            'Referer': 'http://item.jd.com/30110811031.html',
        }

        res = requests.get(url=url, headers=headers)
        data_js = json.loads(res.text)

        return data_js['CommentsCount'][0]['CommentCount']

    def get_apply(self, response):
        """处理适用人群或年龄并返回"""
        apply = re.compile(r"<li.*?>适用[人群年龄]{2}：(.*?)</li>").findall(response.body.decode('GBK', 'ignore'))

        if apply:
            return apply[0]
        else:
            return '无'

    def get_listed_time(self, response):
        """处理上市时间并返回"""
        listed_time = re.compile(r"<li.*?>上市时间：(.*?)</li>").findall(response.body.decode('GBK', 'ignore'))

        if listed_time:
            return listed_time[0]
        else:
            return '无'











